Redukcija dimenzionalnosti¶
In [1]:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from scipy.stats import shapiro, pearsonr, boxcox
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import r2_score
from sklearn.metrics import confusion_matrix
from scipy.spatial.distance import mahalanobis
from scipy.stats import chi2
from sklearn.preprocessing import MinMaxScaler
from sklearn.neighbors import NearestNeighbors
from sklearn.decomposition import PCA, FactorAnalysis
from sklearn.manifold import TSNE
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
import seaborn as sns
Učitavanje podataka o novčanicama
Izvor: Kaggle
In [2]:
data = pd.read_csv('banknotes.csv')
In [3]:
data.head()
Out[3]:
In [4]:
data.shape
Out[4]:
In [5]:
data.describe()
Out[5]:
Razdvajanje regularnih novčanica od falsifikata
In [6]:
regular_notes = data[data.conterfeit == 0]
counterfeit_notes = data[data.conterfeit == 1]
In [7]:
print(f'Regular notes: {regular_notes.shape}')
print(f'Counterfeit notes: {counterfeit_notes.shape}')
Preprocesiranje¶
Analiza raspodele atributa, provera normalnosti i, eventualna, transformacija do normalne raspodele kod regularnih novčanica
Atribut: Length¶
Dužina novčanice
In [8]:
_ = plt.hist(regular_notes.Length)
In [9]:
W, p = shapiro(regular_notes.Length)
# 0.05 -> Kriticna vrednost, za vrednosti manje od 0.05 raspodela nije normalna, sa verovatnocom 0.05.
print(f'Normal distribution? {p > 0.05}, (p={p})')
Atribute: Left¶
Duzina leve strane novčanice
In [10]:
_ = plt.hist(regular_notes.Left)
In [11]:
W, p = shapiro(regular_notes.Left)
# 0.05 -> Kriticna vrednost, za vrednosti manje od 0.05 raspodela nije normalna, sa verovatnocom 0.05.
print(f'Normal distribution? {p > 0.05}, (p={p})')
In [12]:
# Logaritamska transforacija
_, p_log = shapiro(np.log(regular_notes.Left + 1))
print(f'[x -> log(x + 1)]: Normal distribution? {p_log > 0.05}, (p={p_log})\n')
# Transformacija korenovanjem (stepenovanjem)
_, p_sqrt = shapiro(np.sqrt(regular_notes.Left + 1))
print(f'[x -> sqrt(x)]: Normal distribution? {p_sqrt > 0.05}, (p={p_sqrt})\n')
# BOXCOX transformacija
Left_boxcox, Left_lambda = boxcox(regular_notes.Left)
_, p_boxcox = shapiro(Left_boxcox)
print(f'[x -> boxcox(x)]: Normal distribution? {p_boxcox > 0.05}, (p={p_boxcox}, lambda={Left_lambda})')
print(f'Vrednost lambda = {Left_lambda} je prevelika')
Atribut: Right¶
Duzina desne strane novčanice
In [13]:
_ = plt.hist(regular_notes.Right)
In [14]:
W, p = shapiro(regular_notes.Right)
# 0.05 -> Kriticna vrednost, za vrednosti manje od 0.05 raspodela nije normalna, sa verovatnocom 0.05.
print(f'Normal distribution? {p > 0.05}, (p={p})')
In [15]:
# Logaritamska transforacija
_, p_log = shapiro(np.log(regular_notes.Right + 1))
print(f'[x -> log(x + 1)]: Normal distribution? {p_log > 0.05}, (p={p_log})\n')
# Transformacija korenovanjem (stepenovanjem)
_, p_sqrt = shapiro(np.sqrt(regular_notes.Right + 1))
print(f'[x -> sqrt(x)]: Normal distribution? {p_sqrt > 0.05}, (p={p_sqrt})\n')
# BOXCOX transformacija
Right_boxcox, Right_lambda = boxcox(regular_notes.Right)
_, p_boxcox = shapiro(Right_boxcox)
print(f'[x -> boxcox(x)]: Normal distribution? {p_boxcox > 0.05}, (p={p_boxcox}, lambda={Right_lambda})')
print(f'Vrednost lambda = {Right_lambda} je prevelika')
Atribut: Bottom¶
Širina donje margine
In [16]:
_ = plt.hist(regular_notes.Bottom)
In [17]:
W, p = shapiro(regular_notes.Bottom)
# 0.05 -> Kriticna vrednost, za vrednosti manje od 0.05 raspodela nije normalna, sa verovatnocom 0.05.
print(f'Normal distribution? {p > 0.05}, (p={p})')
In [18]:
# Logaritamska transforacija
_, p_log = shapiro(np.log(regular_notes.Bottom + 1))
print(f'[x -> log(x + 1)]: Normal distribution? {p_log > 0.05}, (p={p_log}) <-\n')
# Transformacija korenovanjem (stepenovanjem)
_, p_sqrt = shapiro(np.sqrt(regular_notes.Bottom + 1))
print(f'[x -> sqrt(x)]: Normal distribution? {p_sqrt > 0.05}, (p={p_sqrt})\n')
# BOXCOX transformacija
Bottom_boxcox, Bottom_lambda = boxcox(regular_notes.Bottom)
_, p_boxcox = shapiro(Bottom_boxcox)
print(f'[x -> boxcox(x)]: Normal distribution? {p_boxcox > 0.05}, (p={p_boxcox}, lambda={Bottom_lambda})')
Atribut: Top¶
Širina gornje margine
In [19]:
_ = plt.hist(regular_notes.Top)
In [20]:
W, p = shapiro(regular_notes.Top)
# 0.05 -> Kriticna vrednost, za vrednosti manje od 0.05 raspodela nije normalna, sa verovatnocom 0.05.
print(f'Normal distribution? {p > 0.05}, (p={p})')
In [21]:
# Logaritamska transforacija
_, p_log = shapiro(np.log(regular_notes.Top + 1))
print(f'[x -> log(x + 1)]: Normal distribution? {p_log > 0.05}, (p={p_log})\n')
# Transformacija korenovanjem (stepenovanjem)
_, p_sqrt = shapiro(np.sqrt(regular_notes.Top + 1))
print(f'[x -> sqrt(x)]: Normal distribution? {p_sqrt > 0.05}, (p={p_sqrt})\n')
# BOXCOX transformacija
Top_boxcox, Top_lambda = boxcox(regular_notes.Top)
_, p_boxcox = shapiro(Top_boxcox)
print(f'[x -> boxcox(x)]: Normal distribution? {p_boxcox > 0.05}, (p={p_boxcox}, lambda={Top_lambda}) <--')
Atribut: Diagonal¶
Dužina diagonale novčanice
In [22]:
_ = plt.hist(regular_notes.Diagonal)
In [23]:
W, p = shapiro(regular_notes.Diagonal)
# 0.05 -> Kriticna vrednost, za vrednosti manje od 0.05 raspodela nije normalna, sa verovatnocom 0.05.
print(f'Normal distribution? {p > 0.05}, (p={p})')
In [24]:
# Logaritamska transforacija
_, p_log = shapiro(np.log(regular_notes.Diagonal + 1))
print(f'[x -> log(x + 1)]: Normal distribution? {p_log > 0.05}, (p={p_log})\n')
# Transformacija korenovanjem (stepenovanjem)
_, p_sqrt = shapiro(np.sqrt(regular_notes.Diagonal + 1))
print(f'[x -> sqrt(x)]: Normal distribution? {p_sqrt > 0.05}, (p={p_sqrt})\n')
# BOXCOX transformacija
Diagonal_boxcox, Diagonal_lambda = boxcox(regular_notes.Diagonal)
_, p_boxcox = shapiro(Diagonal_boxcox)
print(f'[x -> boxcox(x)]: Normal distribution? {p_boxcox > 0.05}, (p={p_boxcox}, lambda={Diagonal_lambda}) <--')
Pregled raspodela atributa kod regularnih novcanica i falsifikata¶
Length:
In [25]:
_ = plt.hist(regular_notes.Length)
_ = plt.hist(counterfeit_notes.Length)
Left:
In [26]:
_ = plt.hist(regular_notes.Left)
_ = plt.hist(counterfeit_notes.Left)
Right:
In [27]:
_ = plt.hist(regular_notes.Right)
_ = plt.hist(counterfeit_notes.Right)
log Bottom:
In [28]:
_ = plt.hist(np.log(regular_notes.Right + 1))
_ = plt.hist(np.log(counterfeit_notes.Right + 1))
boxcox Top:
In [29]:
_ = plt.hist(boxcox(regular_notes.Top, lmbda=Top_lambda))
_ = plt.hist(boxcox(counterfeit_notes.Top, lmbda=Top_lambda))
In [30]:
_ = plt.hist(boxcox(regular_notes.Diagonal, lmbda=Diagonal_lambda))
_ = plt.hist(boxcox(counterfeit_notes.Diagonal, lmbda=Diagonal_lambda))
Z Score¶
Napomena: Promenom vrednosti praga sa 3 na niže vrednosti (2.5) dobija se bolji rezultat predikcije (probati). Takodje, korišćenjem netransformisane vrednosti diagonale dobijaju se bolji rezultati jer je raspodela vrlo bliska normalnoj (probati). Ipak, kako formalni uslov normalnosti nije ispunjen, korišćene su vrednosti dobijene BOXCOX transformacijom.
In [31]:
bc_Diagonal = boxcox(regular_notes.Diagonal, lmbda=Diagonal_lambda)
mu = bc_Diagonal.mean()
sigma = bc_Diagonal.std()
print(f'Mean = {mu}, Sigma = {sigma}')
y = [] # Stvarne oznake klase
y_pred = [] # Predvidjene oznake klase
for index, row in data.iterrows():
# Po definiciji BOXCOX transformacije
x = (row.Diagonal**Diagonal_lambda - 1) / Diagonal_lambda
y.append(row.conterfeit)
z = abs(x - mu) / sigma
if z > 3:
y_pred.append(1)
else:
y_pred.append(0)
In [32]:
confusion_matrix(y, y_pred)
Out[32]:
In [33]:
print(f'Score: {r2_score(y, y_pred)}')
Mahalanobisovo rastojanje¶
In [34]:
X = regular_notes.loc[:,['Length','Diagonal']]
X.Diagonal = boxcox(X.Diagonal, lmbda=Diagonal_lambda)
y = []
y_pred = []
mu = X.mean()
V = X.cov()
VI = np.linalg.pinv(V)
p = 0.95
df = X.shape[1]
critical_value = chi2.ppf(p, df)
print(f'Critical value: {critical_value}')
for index, row in data.iterrows():
x = [
row.Left,
(row.Diagonal**Diagonal_lambda - 1) / Diagonal_lambda]
y.append(row.conterfeit)
M = mahalanobis(x, mu, VI)
if M**2 > critical_value:
y_pred.append(1)
else:
y_pred.append(0)
In [35]:
confusion_matrix(y, y_pred)
Out[35]:
In [36]:
print(f'Score: {r2_score(y, y_pred)}')
t-SNE¶
t-SNE se može koristiti samo za vizualizaciju
In [37]:
# Izdvajanje atributa koji nisu oznaka klase
data_attributes = data.iloc[:, 1:] # nulta kolona je oznaka klase
data_attributes.head()
Out[37]:
In [38]:
tsne = TSNE(n_components=2, perplexity=30, n_iter=20000)
tsne_data_transformed = tsne.fit_transform(data_attributes)
In [39]:
X = tsne_data_transformed[:,0]
Y = tsne_data_transformed[:,1]
_ = plt.scatter(X, Y, c=data.conterfeit)
PCA - Principal Component Analysis¶
In [40]:
pca = PCA()
pca_data_transformed = pca.fit_transform(data_attributes)
Analiza udela objasnjene varijanse pomoću svake od kompomenti:
In [41]:
evr = pca.explained_variance_ratio_
for i in range(evr.shape[0]):
print(f'PC{i + 1}: {evr[i] * 100}%')
_ = plt.bar([i + 1 for i in range(evr.shape[0])], evr)
Pomoću prve dve komponente može se objasniti ~87% udela varijanse
In [42]:
X = pca_data_transformed[:,0]
Y = pca_data_transformed[:,1]
_ = plt.scatter(X, Y, c=data.conterfeit)
Izračunavanje komponenti vršiće se na osnovu regularnih novčanica, kako falsifikati ne bi imali uticaj u proceni parametara raspodele. Ovo će biti vršeno na navedeni način samo u ovom specifičnom slučaju prepoznavanja elemenata van granica, u slučaju klasifikacije biće transformisan ceo skup podataka.
In [43]:
pca_regular_data_transformed = pca.fit_transform(regular_notes.iloc[:,1:])
Pomocu prve tri komponente može se objasniti 85% udela varijanse
In [44]:
evr = pca.explained_variance_ratio_
for i in range(evr.shape[0]):
print(f'PC{i + 1}: {evr[i] * 100}%')
_ = plt.bar([i + 1 for i in range(evr.shape[0])], evr)
Provera normalnosti prve tri komponente:
In [45]:
_ = plt.hist(pca_regular_data_transformed[:,0])
_, p = shapiro(pca_regular_data_transformed[:,0])
print(f'Normal distribution? {p > 0.05}')
In [46]:
log_const = 5 # Konstanta, kako vrednosti ne bi bile negativne za potrebe logaritma
plt.hist(np.log(pca_regular_data_transformed[:,0] + log_const))
_, p_log = shapiro(np.log(pca_regular_data_transformed[:,0] + log_const))
print(f'PC1: Normal distribution? {p_log > 0.05}')
In [47]:
_ = plt.hist(pca_regular_data_transformed[:,1])
_, p = shapiro(pca_regular_data_transformed[:,1])
print(f'PC2: Normal distribution? {p > 0.05}')
In [48]:
log_const = 5 # Konstanta, kako vrednosti ne bi bile negativne za potrebe logaritma
# Napomena: Konstanta ima uticaj na oblik raspodele tako da treba biti pazljiv sa vrednostima
# sto manja vrednost - to manji uticaj
plt.hist(np.log(pca_regular_data_transformed[:,1] + log_const))
_, p_log = shapiro(np.log(pca_regular_data_transformed[:,1] + log_const))
print(f'Normal distribution? {p_log > 0.05}')
In [49]:
_ = plt.hist(pca_regular_data_transformed[:,2])
_, p = shapiro(pca_regular_data_transformed[:,2])
print(f'PC2: Normal distribution? {p > 0.05}')
In [50]:
pca_all_data_transformed = pca.transform(data_attributes)
pca_all_data_transformed_df = pd.DataFrame(data=pca_all_data_transformed[:,:3], columns=['PC1','PC2','PC3'])
# Transformacije to normalne raspodele
pca_all_data_transformed[:,0] = np.log(pca_all_data_transformed[:,0] + log_const)
pca_all_data_transformed[:,1] = np.log(pca_all_data_transformed[:,1] + log_const)
In [51]:
X = pd.DataFrame(data=pca_all_data_transformed[:100,:3], columns=['PC1','PC2','PC3'])
X.head()
Out[51]:
In [52]:
y = []
y_pred = []
mu = X.mean()
V = X.cov()
VI = np.linalg.pinv(V)
p = 0.95
df = X.shape[1]
critical_value = chi2.ppf(p, df)
print(f'Critical value: {critical_value}')
for index, row in pca_all_data_transformed_df.iterrows():
x = row
y.append(int(data.iloc[index,:].conterfeit))
M = mahalanobis(x, mu, VI)
if M**2 > critical_value:
y_pred.append(1)
else:
y_pred.append(0)
In [53]:
confusion_matrix(y, y_pred)
Out[53]:
In [54]:
r2_score(y, y_pred)
Out[54]:
Redukcija za potrebe klasifikacije¶
LDA - Linear Discriminant Analysis¶
In [55]:
lda = LinearDiscriminantAnalysis()
lda_data_transformed = lda.fit_transform(data_attributes, data.conterfeit) # atributi, oznake klase
In [56]:
lda_data_transformed
_ = plt.scatter(lda_data_transformed, [0 for i in range(lda_data_transformed.shape[0])], c=data.conterfeit) # Podaci su jednodimenzioni
U ovom, trivijalnom slučaju, klasifikacija se može vršiti jednostavnim stablom dubine 1 (jedno pitanje):
In [57]:
y = []
y_pred = []
for i in range(lda_data_transformed.shape[0]):
x = lda_data_transformed[i]
if i < 100:
y.append(0)
else:
y.append(1)
if x[0] < 0:
y_pred.append(0)
else:
y_pred.append(1)
In [58]:
confusion_matrix(y, y_pred)
Out[58]:
In [59]:
r2_score(y, y_pred)
Out[59]:
In [60]:
num_components = data_attributes.shape[1]
mdl = FactorAnalysis(n_components=num_components)
fa_data_transformed = mdl.fit_transform(data_attributes)
In [61]:
fa_data_transformed = pd.DataFrame(data = fa_data_transformed, columns=[f'Factor {i + 1}' for i in range(num_components)])
fa_data_transformed.head()
Out[61]:
Može se uočiti da je dovoljno sačuvati samo prva tri faktora
Koeficijenti faktora u atributima:
In [62]:
pd.DataFrame(data=mdl.components_, columns=data_attributes.columns).head()
Out[62]:
Uporediti dobijene koeficijente sa korelacijama medju atributima:
In [63]:
_ = sns.heatmap(data_attributes.corr())
